
sh run_pretrain_megatron_llama.sh  \
dsw  \
../../ \
7B   \
1    \
8 \
1e-5   \
1e-6   \
128  \
128  \
0   \
bf16  \
8   \
1  \
sel  \
true   \
false  \
false   \
false   \
100000  \
/mnt/workspace/llama2-datasets/wudao_llamabpe_text_document   \
/mnt/workspace/llama2-ckpts/Llama-2-7b-hf   \
100000000   \
10000   \
/mnt/workspace/output_run_mcore_llama2_ppu